package com.jsoup;

import java.io.IOException;
import org.apache.log4j.Logger;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.TaoBaoHtmlConstant;
import com.util.UtilsFile;
import com.util.UtilsWeb;
import com.util.UtilsXML;

public class TaobaoSpider {
	private static Logger log = Logger.getLogger(TaobaoSpider.class);
	private static String defaultPath = "淘宝xml";
	private static String apiName;
	private static String apiMethod;
	private static String apiUrl = "http://open.taobao.com/doc/category_list.htm?id=102";
	/** 在xml头部插入String，为0则为不插入*/
	private static String insertHead = "<b2b>";
	/** 在xml尾部插入String，为0则为不插入*/
	private static String insertTail = "</b2b>";
	/** 设置请求等待时间，网速慢的情况下等待时间设大点*/
	private static final int WAIT_TIME = 300;
	/** 是否格式化xml后再输入到文件*/
	private static final boolean TO_FORMAT = true;
	
	public TaobaoSpider(){
		log.info("正在搜索taobao api......");
	}
	
	public TaobaoSpider(String apiUrl){
		TaobaoSpider.apiUrl = apiUrl;
		log.info("正在搜索taobao api......");
	}
	
	/**
	 * 得到所有api列表
	 * @param url
	 * @throws IOException
	 * @throws InterruptedException
	 */
	public void getApiList() throws IOException, InterruptedException{
//		Document doc1 = null;
//		try {
//			Connection connect = Jsoup.connect(apiUrl);
//			connect.timeout(3000);
//			doc1 = connect.get();
//		} catch (IOException e) {
//			log.info("请求数据失败！重试......" + e.getMessage());
//			getApiList(); 
//			return;
//		}
		
		String webHtml = UtilsWeb.getWebHtml(apiUrl);
		Document doc1 = Jsoup.parse(webHtml);

		
		/** 线程sleep一会， 等待请求完毕*/
	//	Thread.sleep(WAIT_TIME);
		
		/** 得到所有的api链接地址  */
		Elements elements = doc1.getElementsByClass(TaoBaoHtmlConstant.API_LIST_CLASS_NAME).select("a");
		Element element = null;
		int size = elements.size();
		log.info("淘宝总开放 " + size + " 个API.");
		for (int i = 0; i < size; i++) {
			element = elements.get(i);
			String text = element.text();
			String href = element.attr("href");
			apiName = text;
			log.info("---- "+(i+1)+". "+ text+" -------");
			getApiMethodlist(href);
		}
		
	}
	
	/**
	 * 得到所有api method列表
	 * @param url
	 * @throws IOException
	 * @throws InterruptedException
	 */
	public void getApiMethodlist(String url) throws IOException, InterruptedException{
//		Document doc1 = null;
//		try {
//			Connection connect = Jsoup.connect(url);
//			connect.timeout(3000);
//			doc1 = connect.get();
//		} catch (IOException e) {
//			log.info("请求数据失败！重试......" + e.getMessage());
//			getApiMethodlist(url); 
//			return;
//		}
		
//		Thread.sleep(WAIT_TIME);
		String webHtml = UtilsWeb.getWebHtml(url);
		Document doc1 = Jsoup.parse(webHtml);
		
		/** 得到所有的api列表  */
		Element apiElement = doc1.getElementsByClass("api-list").first();
		
		/** 得到所有的api链接地址  */
		Elements elements = apiElement.select("a");
		Element element = null;
		int size = elements.size();
		log.info("此api共有 " + size + " 个方法.");
		for (int i = 0; i < size; i++) {
			element = elements.get(i);
			String text = element.text();
			String href = element.attr("href");
			if(text.trim().length()>1){
				apiMethod = text;
				log.info("---- "+(i+1)+"). "+ text+" -------");
				getData3(href);
			}
		}
	}
	
	/**
	 * 得到最终request和response的 xml，并写入到文件
	 * @param url
	 * @throws IOException
	 * @throws InterruptedException
	 */
	public void getData3(String url) throws IOException, InterruptedException{
//		Document doc1 = null;
//		try {
//			Connection connect = Jsoup.connect(url);
//			connect.followRedirects(false);
//			connect.timeout(3000);
//			doc1 = connect.get();
//		} catch (IOException e) {
//			log.info("请求数据失败！重试......" + e.getMessage());
//			getData3(url); 
//			return;
//		}
//		
//		Thread.sleep(600);
		String webHtml = UtilsWeb.getWebHtml(url);
		Document doc1 = Jsoup.parse(webHtml);
		
		String title = doc1.getElementsByTag("title").get(0).html();
		if(title.equals("302 Found")){
			log.error("请求api字段时发生异常！302 Found,程序将放弃302重定向。正在重新请求......");
			getData3(url);
			return;
		}
		
		/**
		 * 得到xml根节点的名字
		 */
		Elements apiElement2 = doc1.select("#example ~ .img-wrap");
		Element select2 = apiElement2.select("pre").first();
		String xml = select2.text();
		String rootTag = getRootTag(xml);
		StringBuffer sb = new StringBuffer();
		StringBuffer responseSb = new StringBuffer();
		rootTag = rootTag.replace("response", "request");
		sb.append("<" + rootTag + ">");
		
		
		/** 得到所有的api应用级输入参数字段  */ 
		Elements apiElement = doc1.select("#api-param ~ table tbody");
		Elements select = apiElement.select("tr");
		
		for (Element element : select) {
			Elements eles = element.select("td");
			String fields = eles.get(0).text();
			String type = eles.get(1).text();
			String must = eles.get(2).text();
			String examples = eles.get(3).text();
			String des = eles.get(5).text();
			sb.append( "<!-- 是否必须:["+must+"], 类型:[" +type+ "], 描述:[" +des+ "] -->"+"<" + fields +">" + examples + " </" + fields +">");
		}
		sb.append("</" + rootTag + ">");
		
		
		/** 得到所有的api返回结果字段 */
		rootTag = rootTag.replace("request", "response");
		responseSb.append("<" + rootTag + ">");
		responseSb.append(getResponseFields(doc1, "#result ~ table tbody"));
		responseSb.append("</" + rootTag + ">");
		
		
		/** 是否在头和尾插入*/
		if (!insertHead.equals("0") && !insertTail.equals("0")) {
			sb.insert(0, insertHead).append(insertTail);
			responseSb.insert(0, insertHead).append(insertTail);
		}
		
		/** 记录日志 */
		log.info("request  xml: " + sb.toString());
		log.info("response xml: " + responseSb.toString());
	
		String requestXml = "";
		String responseXml = "";
		/** 格式化xml */ 
		if (TO_FORMAT) {
				try {
					requestXml = UtilsXML.formatXML(sb.toString());
					responseXml = UtilsXML.formatXML(responseSb.toString());
				} catch (Exception e) {
					log.error("格式化xml异常！ "+e.getMessage());
					requestXml = sb.toString();
					responseXml = responseSb.toString();
				}
		}else{
			requestXml = sb.toString();
			responseXml = responseSb.toString();
		}
		
		/** 写入到文件 */
		UtilsFile.writeToFile("../"+defaultPath+"/"+apiName+"/"+apiMethod+".request.xml",  true, requestXml);
		UtilsFile.writeToFile("../"+defaultPath+"/"+apiName+"/"+apiMethod+".response.xml", true, responseXml);

	}
	
	/**
	 * 得到api字段
	 * @param doc1
	 * @param selectExpression
	 * @return
	 * @throws InterruptedException
	 * @throws IOException
	 */
	public StringBuffer getResponseFields(Document doc1, String selectExpression) throws InterruptedException, IOException{
		StringBuffer resultSb = new StringBuffer();
		/** 得到所有的api返回结果字段 */
		Elements apiElement = doc1.select(selectExpression);
		Elements select = apiElement.select("tr");
		for (Element element : select) {
			Elements eles = element.select("td");
			String fields = eles.get(0).text().trim();
			String type = eles.get(1).text();
			String must = eles.get(2).text();
			String examples = eles.get(3).text();
			String des = eles.get(4).text().replaceAll("<<", " ").replaceAll(">>", " ").replaceAll(">", " ").replaceAll("<", " ");
			resultSb.append( "<!-- 是否必须:["+must+"], 类型:[" +type+ "], 描述:[" +des+ "] -->"+"<" + fields +">");
			if( !fields.equals("Boolean") && 
					!fields.equals("Date") && 
					!fields.equals("String") && 
					!fields.equals("Number") && 
					!fields.equals("byte[]") && 
					!fields.equals("Field List") && 
					!fields.equals("Price") && 
					!fields.equals("Boolean[]") && 
					!fields.equals("Date[]") &&
					!fields.equals("String[]") &&
					!fields.equals("Number[]") &&
					!fields.equals("Price[]")){
				
				String typeHref = eles.get(1).getElementsByTag("a").attr("href");
				resultSb.append(getConstructList(typeHref));
				resultSb.append("</" + fields +">");
			}else{
				resultSb.append("<" + fields +">" + examples + " </" + fields +">");
			}
			
		}
		
		return resultSb;
	}
	
	/**
	 * 得到结构体的所有字段
	 * @param url
	 * @return
	 * @throws IOException
	 * @throws InterruptedException
	 */
	private StringBuffer getConstructList(String url) throws IOException, InterruptedException {
		url = "http://api.taobao.com/apidoc/" + url;
//		Document doc1 = null;
//		try {
//			Connection connect = Jsoup.connect(url1);
//			connect.followRedirects(false);
//			connect.timeout(3000);
//			doc1 = connect.get();
//		} catch (IOException e) {
//			log.info("请求数据失败！重试......" + e.getMessage());
//			getConstructList(url); 
//			return null;
//		}
		String webHtml = UtilsWeb.getWebHtml(url);
		Document doc1 = Jsoup.parse(webHtml);
		
//		Thread.sleep(WAIT_TIME);
		
		return getResponseFields(doc1, ".api-table tbody");
	}

	/**
	 * 得到response的根标签的名字
	 * @param xml
	 * @return
	 */
	private String getRootTag(String xml) {
		String name;
		try {
			xml = xml.replace("<?xml version=\"1.0\"encoding=\"utf-8\"?>", "");
			name = xml.substring(xml.indexOf("<")+1, xml.indexOf(">"));
		} catch (StringIndexOutOfBoundsException e) {
			log.error("从xml中解析根标签名称时异常!");
			return xml;
		}

		return name;
	}
}
